#This script can be directly run from the command line.
#It trains on a set amount of the training data and then scores each post in the leaderboard data.

import sys
import io
import re
import shlex
import Parser
import Post
import StatusTotals
import WordProcessor
import WordBag
import FileIO
import OversamplingWeights
import NaiveBayes

class Controls:
	NUMBER_OF_POSTS_TO_TRAIN_ON = 1500000
	NUMBER_FROM_EACH_CLASS = 4878

def main():
	trainingFileStream = FileIO.openTrainingFile(FileIO.READ_MODE)
	wordBag = WordBag.WordBag()
	totals = StatusTotals.StatusTotals()
	
	print "Training Start..."
	#Train
	count = 0
	while (not Parser.isEndOfFile(trainingFileStream)):
		newPost = Parser.readPost(trainingFileStream)
		postStatus =  newPost.openStatus

		totals.increment(postStatus)
		WordProcessor.fillWordBag(newPost, wordBag)
		print str(totals.getTotal())
		#Drop the least occuring words to keep memory usage low
		if(totals.getTotal()%NaiveBayes.Controls.DROP_WORDS_PERIOD==0):
			wordBag.dropLeastOccuringWords(NaiveBayes.Controls.DROP_WORDS_THRESHOLD)
		count += 1
		if (count >= Controls.NUMBER_OF_POSTS_TO_TRAIN_ON):
			break
	print "Scoring Start..."
	#Print Scores
	leaderboardFileStream = FileIO.openLeaderboardFile(FileIO.READ_MODE)
	resultsFileStream = FileIO.openResultsFile(FileIO.WRITE_MODE)
	Parser.simpleReadHeader(leaderboardFileStream)
	numZeroes = 0
	lines = 0
	#while (not Parser.isEndOfFile(leaderboardFileStream)):
	while(True):
		leaderboardPost = Parser.readLeaderboardPost(leaderboardFileStream)
		openPrediction = NaiveBayes.predictPost(leaderboardPost, wordBag, totals, Post.PostStatusEnum.OPEN)
		offTopicPrediction = NaiveBayes.predictPost(leaderboardPost, wordBag, totals, Post.PostStatusEnum.OFF_TOPIC)
		notConstructivePrediction = NaiveBayes.predictPost(leaderboardPost, wordBag, totals, Post.PostStatusEnum.NOT_CONSTRUCTIVE)
		notARealQuestionPrediction = NaiveBayes.predictPost(leaderboardPost, wordBag, totals, Post.PostStatusEnum.NOT_A_REAL_QUESTION)
		tooLocalizedPrediction = NaiveBayes.predictPost(leaderboardPost, wordBag, totals, Post.PostStatusEnum.TOO_LOCALIZED)
		totalPrediction = openPrediction + offTopicPrediction + notConstructivePrediction + notARealQuestionPrediction + tooLocalizedPrediction
		
		#print str(openPrediction) + " " +str(offTopicPrediction) + " " +str(notConstructivePrediction) + " " +str(notARealQuestionPrediction) + " " +str(tooLocalizedPrediction)
		if(totalPrediction == 0):
			print str(numZeroes)
			numZeroes +=1
			openPrediction += float(.0001)
			offTopicPrediction += float(.0001) 
			notConstructivePrediction += float(.0001)
			notARealQuestionPrediction += float(.0001)
			tooLocalizedPrediction += float(.0001)
			totalPrediction += (5*float(.0001))
		
		resultsFileStream.write(str(float(notARealQuestionPrediction) / float(totalPrediction))+","+str(float(notConstructivePrediction) / float(totalPrediction))+","+str(float(offTopicPrediction) / float(totalPrediction))+","+str(float(openPrediction) / float(totalPrediction))+","+str(float(tooLocalizedPrediction) / float(totalPrediction))+"\n")
		lines+=1
		print str(lines) + " " + str(leaderboardPost.postID)
		if(lines>=73290):
			break
		
	resultsFileStream.close()
	
#Use Equal training data from each class
def old():
	trainingFileStream = FileIO.openTrainingFile(FileIO.READ_MODE)
	wordBag = WordBag.WordBag()
	totals = StatusTotals.StatusTotals()
	
	print "Training Start..."
	#Train
	count = 0
	while (not Parser.isEndOfFile(trainingFileStream)):
		newPost = Parser.readPost(trainingFileStream)
		postStatus =  newPost.openStatus
		if(totals.getCountByStatus(postStatus) < Controls.NUMBER_FROM_EACH_CLASS):
			totals.increment(postStatus)
			WordProcessor.fillWordBag(newPost, wordBag)
			print str(totals.getTotal())
			#Drop the least occuring words to keep memory usage low
			if(totals.getTotal()%NaiveBayes.Controls.DROP_WORDS_PERIOD==0):
				wordBag.dropLeastOccuringWords(NaiveBayes.Controls.DROP_WORDS_THRESHOLD)
		count += 1
		if (count >= Controls.NUMBER_OF_POSTS_TO_TRAIN_ON):
			break
	print "Scoring Start..."
	#Print Scores
	leaderboardFileStream = FileIO.openLeaderboardFile(FileIO.READ_MODE)
	resultsFileStream = FileIO.openResultsFile(FileIO.WRITE_MODE)
	Parser.readHeader(leaderboardFileStream)
	numZeroes = 0
	while (not Parser.isEndOfFile(leaderboardFileStream)):
	
		leaderboardPost = Parser.readLeaderboardPost(leaderboardFileStream)
		openPrediction = NaiveBayes.predictPost(leaderboardPost, wordBag, totals, Post.PostStatusEnum.OPEN)
		offTopicPrediction = NaiveBayes.predictPost(leaderboardPost, wordBag, totals, Post.PostStatusEnum.OFF_TOPIC)
		notConstructivePrediction = NaiveBayes.predictPost(leaderboardPost, wordBag, totals, Post.PostStatusEnum.NOT_CONSTRUCTIVE)
		notARealQuestionPrediction = NaiveBayes.predictPost(leaderboardPost, wordBag, totals, Post.PostStatusEnum.NOT_A_REAL_QUESTION)
		tooLocalizedPrediction = NaiveBayes.predictPost(leaderboardPost, wordBag, totals, Post.PostStatusEnum.TOO_LOCALIZED)
		totalPrediction = openPrediction + offTopicPrediction + notConstructivePrediction + notARealQuestionPrediction + tooLocalizedPrediction
		
		#print str(openPrediction) + " " +str(offTopicPrediction) + " " +str(notConstructivePrediction) + " " +str(notARealQuestionPrediction) + " " +str(tooLocalizedPrediction)
		if(totalPrediction == 0):
			print str(numZeroes)
			numZeroes +=1
			openPrediction += float(.0001)
			offTopicPrediction += float(.0001) 
			notConstructivePrediction += float(.0001)
			notARealQuestionPrediction += float(.0001)
			tooLocalizedPrediction += float(.0001)
			totalPrediction += (5*float(.0001))
		
		resultsFileStream.write(str(float(notARealQuestionPrediction) / float(totalPrediction))+","+str(float(notConstructivePrediction) / float(totalPrediction))+","+str(float(offTopicPrediction) / float(totalPrediction))+","+str(float(openPrediction) / float(totalPrediction))+","+str(float(tooLocalizedPrediction) / float(totalPrediction))+"\n")
	
	resultsFileStream.close()
main()